{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from __future__ import print_function, division\n",
      "import pandas as pd\n",
      "import numpy as np\n",
      "import itertools, os, time\n",
      "\n",
      "print(\"Pandas version = \", pd.__version__)\n",
      "print(\"Numpy version =  \", np.__version__)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Pandas version =  0.13.1\n",
        "Numpy version =   1.8.1\n"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Create some toy data"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "N_PERIODS = 1E4\n",
      "N_METERS = 5\n",
      "N_MEASUREMENTS_PER_METER = 3\n",
      "\n",
      "meters = ['meter{:d}'.format(i) for i in range(1,N_METERS+1)]\n",
      "meters = [[m]*N_MEASUREMENTS_PER_METER for m in meters]\n",
      "flatten_2d_list = lambda lst: list(itertools.chain(*lst))\n",
      "meters = flatten_2d_list(meters)\n",
      "level2 = ['power', 'power', 'voltage'][:N_MEASUREMENTS_PER_METER] * N_METERS\n",
      "level3 = ['active', 'reactive', ''][:N_MEASUREMENTS_PER_METER] * N_METERS\n",
      "\n",
      "\n",
      "columns = [meters, level2, level3]\n",
      "columns = pd.MultiIndex.from_arrays(columns)\n",
      "rng = pd.date_range('2012-01-01', freq='S', periods=N_PERIODS)\n",
      "data = np.random.randint(low=0, high=1000, \n",
      "                         size=(N_PERIODS, \n",
      "                               N_METERS*N_MEASUREMENTS_PER_METER))\n",
      "df = pd.DataFrame(data=data, index=rng, columns=columns, dtype=np.float32)\n",
      "\n",
      "df.ix[:3]\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr>\n",
        "      <th></th>\n",
        "      <th colspan=\"3\" halign=\"left\">meter1</th>\n",
        "      <th colspan=\"3\" halign=\"left\">meter2</th>\n",
        "      <th colspan=\"3\" halign=\"left\">meter3</th>\n",
        "      <th colspan=\"3\" halign=\"left\">meter4</th>\n",
        "      <th colspan=\"3\" halign=\"left\">meter5</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th></th>\n",
        "      <th colspan=\"2\" halign=\"left\">power</th>\n",
        "      <th>voltage</th>\n",
        "      <th colspan=\"2\" halign=\"left\">power</th>\n",
        "      <th>voltage</th>\n",
        "      <th colspan=\"2\" halign=\"left\">power</th>\n",
        "      <th>voltage</th>\n",
        "      <th colspan=\"2\" halign=\"left\">power</th>\n",
        "      <th>voltage</th>\n",
        "      <th colspan=\"2\" halign=\"left\">power</th>\n",
        "      <th>voltage</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th></th>\n",
        "      <th>active</th>\n",
        "      <th>reactive</th>\n",
        "      <th></th>\n",
        "      <th>active</th>\n",
        "      <th>reactive</th>\n",
        "      <th></th>\n",
        "      <th>active</th>\n",
        "      <th>reactive</th>\n",
        "      <th></th>\n",
        "      <th>active</th>\n",
        "      <th>reactive</th>\n",
        "      <th></th>\n",
        "      <th>active</th>\n",
        "      <th>reactive</th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>2012-01-01 00:00:00</th>\n",
        "      <td>  54</td>\n",
        "      <td> 778</td>\n",
        "      <td> 352</td>\n",
        "      <td> 312</td>\n",
        "      <td> 585</td>\n",
        "      <td> 525</td>\n",
        "      <td> 403</td>\n",
        "      <td> 954</td>\n",
        "      <td> 688</td>\n",
        "      <td> 937</td>\n",
        "      <td> 213</td>\n",
        "      <td> 374</td>\n",
        "      <td> 681</td>\n",
        "      <td> 317</td>\n",
        "      <td> 171</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2012-01-01 00:00:01</th>\n",
        "      <td> 469</td>\n",
        "      <td> 868</td>\n",
        "      <td> 815</td>\n",
        "      <td> 603</td>\n",
        "      <td> 781</td>\n",
        "      <td> 388</td>\n",
        "      <td> 417</td>\n",
        "      <td> 960</td>\n",
        "      <td>  99</td>\n",
        "      <td> 701</td>\n",
        "      <td> 628</td>\n",
        "      <td> 393</td>\n",
        "      <td> 488</td>\n",
        "      <td> 133</td>\n",
        "      <td>  46</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2012-01-01 00:00:02</th>\n",
        "      <td> 305</td>\n",
        "      <td> 399</td>\n",
        "      <td>  28</td>\n",
        "      <td> 399</td>\n",
        "      <td> 622</td>\n",
        "      <td> 155</td>\n",
        "      <td> 554</td>\n",
        "      <td> 936</td>\n",
        "      <td> 492</td>\n",
        "      <td> 487</td>\n",
        "      <td> 188</td>\n",
        "      <td> 730</td>\n",
        "      <td> 340</td>\n",
        "      <td> 154</td>\n",
        "      <td> 650</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "<p>3 rows \u00d7 15 columns</p>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 16,
       "text": [
        "                     meter1                     meter2                     \\\n",
        "                      power            voltage   power            voltage   \n",
        "                     active  reactive           active  reactive            \n",
        "2012-01-01 00:00:00      54       778      352     312       585      525   \n",
        "2012-01-01 00:00:01     469       868      815     603       781      388   \n",
        "2012-01-01 00:00:02     305       399       28     399       622      155   \n",
        "\n",
        "                     meter3                     meter4                     \\\n",
        "                      power            voltage   power            voltage   \n",
        "                     active  reactive           active  reactive            \n",
        "2012-01-01 00:00:00     403       954      688     937       213      374   \n",
        "2012-01-01 00:00:01     417       960       99     701       628      393   \n",
        "2012-01-01 00:00:02     554       936      492     487       188      730   \n",
        "\n",
        "                     meter5                     \n",
        "                      power            voltage  \n",
        "                     active  reactive           \n",
        "2012-01-01 00:00:00     681       317      171  \n",
        "2012-01-01 00:00:01     488       133       46  \n",
        "2012-01-01 00:00:02     340       154      650  \n",
        "\n",
        "[3 rows x 15 columns]"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Save the data to an HDF5 file"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "       \n",
      "envi_r = {'manufacturer': 'Current Cost', \n",
      "          'name': 'EnviR', \n",
      "          'sample_period': 6}\n",
      "\n",
      "filename = 'random.h5'\n",
      "store = pd.HDFStore(filename, 'w', complevel=9, complib='bzip2')\n",
      "# store.root._v_attrs.meter_types = {envi_r['model']: envi_r}\n",
      "for meter in df.columns.levels[0]:\n",
      "    print(\"Saving \" + meter)\n",
      "    key = 'building1/electric/' + meter\n",
      "    store.put(key, df[meter], format='table')\n",
      "    store.get_storer(key).attrs.metadata = {'model': envi_r['name'], 'submeter_of': 1}\n",
      "    print(store.get_storer(key).attrs.metadata)\n",
      "print(store)\n",
      "\n",
      "# Save dataset-wide metadata\n",
      "# store._handle.create_group('/', 'nilm_metadata', 'NILM Metadata')\n",
      "# store.root.nilm_metadata._v_attrs.meter_types = {envi_r['model']: envi_r}\n",
      "\n",
      "store.root._v_attrs.dataset = {'meter_models': {envi_r['name']: envi_r}}\n",
      "\n",
      "store.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Saving meter1\n",
        "{'model': 'EnviR', 'submeter_of': 1}\n",
        "Saving meter2\n",
        "{'model': 'EnviR', 'submeter_of': 1}"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Saving meter3\n",
        "{'model': 'EnviR', 'submeter_of': 1}\n",
        "Saving meter4\n",
        "{'model': 'EnviR', 'submeter_of': 1}"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Saving meter5\n",
        "{'model': 'EnviR', 'submeter_of': 1}\n",
        "<class 'pandas.io.pytables.HDFStore'>\n",
        "File path: random.h5\n",
        "/building1/electric/meter1            frame_table  (typ->appendable,nrows->10000,ncols->3,indexers->[index])\n",
        "/building1/electric/meter2            frame_table  (typ->appendable,nrows->10000,ncols->3,indexers->[index])\n",
        "/building1/electric/meter3            frame_table  (typ->appendable,nrows->10000,ncols->3,indexers->[index])\n",
        "/building1/electric/meter4            frame_table  (typ->appendable,nrows->10000,ncols->3,indexers->[index])\n",
        "/building1/electric/meter5            frame_table  (typ->appendable,nrows->10000,ncols->3,indexers->[index])"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 17
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "filename = 'random.h5'\n",
      "store = pd.HDFStore(filename, 'r')\n",
      "print(store['building1/electric/meter1'].ix[:5])\n",
      "print(store.get_storer('building1/electric/meter1').attrs.metadata)\n",
      "print(store.get_storer('building1/electric/meter2').attrs.metadata)\n",
      "\n",
      "print(store.root._v_attrs.dataset['meter_models'])\n",
      "\n",
      "store.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "                      power            voltage\n",
        "                     active  reactive         \n",
        "2012-01-01 00:00:00      54       778      352\n",
        "2012-01-01 00:00:01     469       868      815\n",
        "2012-01-01 00:00:02     305       399       28\n",
        "2012-01-01 00:00:03     478       457      926\n",
        "2012-01-01 00:00:04     885        95      396\n",
        "\n",
        "[5 rows x 3 columns]\n",
        "{'model': 'EnviR', 'submeter_of': 1}\n",
        "{'model': 'EnviR', 'submeter_of': 1}\n",
        "{'EnviR': {'manufacturer': 'Current Cost', 'name': 'EnviR', 'sample_period': 6}}"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "class TableStore(object):\n",
      "    \"\"\"Provides a common interface to all physical data stores.  \n",
      "    Supports hierarchical stores.  Only reads data.  Reads and writes metadata.\n",
      "    e.g. one HDF5 file, or one Xively feed, or one REDD data set etc.\n",
      "    There could be subclasses for HDF5, Xively feeds,\n",
      "    Current Cost meters, OpenEnergyMonitor, rfm_ecomanager,\n",
      "    REDD, UKPD, etc. Or MetOffice XLS data, \n",
      "    weather feeds from the internet etc.\n",
      "    \n",
      "    Only opens the underlying data source when absolutely necessary \n",
      "    (by minimising the time the data source is open, we can minimise\n",
      "    the chance of data being corrupted if another process changes it).\n",
      "    \"\"\"\n",
      "    pass\n",
      "\n",
      "MAX_MEM_ALLOWANCE_IN_BYTES = 1E9\n",
      "\n",
      "def date_range_to_terms(start_date=None, end_date=None):\n",
      "    terms = []\n",
      "    if start_date is not None:\n",
      "        terms.append(\"index>=start_date\")\n",
      "    if end_date is not None:\n",
      "        terms.append(\"index<end_date\")\n",
      "    return terms\n",
      "\n",
      "class HDFTableStore(TableStore):\n",
      "    def __init__(self, filename, table, start_date=None, end_date=None):\n",
      "        \"\"\"\n",
      "        Parameters\n",
      "        ----------\n",
      "        filename : string\n",
      "        table : string\n",
      "            e.g. '/building1/utility/electric/meter1'\n",
      "        start_end, end_date : string or pd.Timestamp, optional\n",
      "            Defines a 'mask' applied to all operations on this object.\n",
      "        \"\"\"\n",
      "        self.store = pd.HDFStore(filename, 'r')\n",
      "        self.store.close()\n",
      "        self._check_key(table)\n",
      "        self.table = table\n",
      "        self.start_date = pd.Timestamp(start_date) if start_date else None\n",
      "        self.end_date = pd.Timestamp(end_date) if end_date else None\n",
      "               \n",
      "    def load(self, cols=None, start_date=None, end_date=None):\n",
      "        \"\"\"\n",
      "        Parameters\n",
      "        ----------\n",
      "        cols : list or 'index', optional\n",
      "            e.g. [('power', 'active'), ('power', 'reactive'), ('voltage', '')]\n",
      "            if not provided then will return all columns from the table.\n",
      "        start_date, end_date : string or pd.Timestamp, optional\n",
      "            defines the time period to load as range (start_date, end_date]\n",
      "            \n",
      "        Returns\n",
      "        -------\n",
      "        if `cols=='index'` then returns a pd.DatetimeIndex\n",
      "        else returns a pd.DataFrame\n",
      "        \"\"\"\n",
      "        start_date, end_date = self.restrict_start_and_end_dates(start_date, end_date)\n",
      "        \n",
      "        # Create list of query terms\n",
      "        terms = date_range_to_terms(start_date, end_date)\n",
      "        if cols is not None:\n",
      "            if not self.table_has_column_names(cols):\n",
      "                raise KeyError('at least one of ' + str(cols) + ' is not a valid column')\n",
      "            terms.append(\"columns==cols\")\n",
      "        if terms == []:\n",
      "            terms = None\n",
      "        \n",
      "        # Check we won't use too much memory\n",
      "        mem_requirement = self.estimate_memory_requirement(cols, start_date, end_date)\n",
      "        if mem_requirement > MAX_MEM_ALLOWANCE_IN_BYTES:\n",
      "            raise MemoryError('Requested data would use too much memory.')\n",
      "        \n",
      "        # Read data\n",
      "        self.store.open()\n",
      "        data = self.store.select(key=self.table, where=terms, auto_close=True)\n",
      "        if cols == 'index':\n",
      "            data = data.index\n",
      "        return data\n",
      "    \n",
      "    def table_has_column_names(self, cols):\n",
      "        \"\"\"\n",
      "        Parameters\n",
      "        ----------\n",
      "        cols : string or list of strings\n",
      "        \n",
      "        Returns\n",
      "        -------\n",
      "        boolean\n",
      "        \"\"\"\n",
      "        if isinstance(cols, str):\n",
      "            cols = [cols]\n",
      "        query_cols = set(cols)\n",
      "        table_cols = set(self.column_names + ['index'])\n",
      "        return query_cols.issubset(table_cols)\n",
      "    \n",
      "    def get_generator(self, periods=None, cols=None):\n",
      "        \"\"\"\n",
      "        Parameters\n",
      "        ----------\n",
      "        periods : list of (start_date, end_date) tuples, optional\n",
      "            e.g. [(\"2013-01-01\", \"2013-02-01\"), (\"2013-02-01\", \"2013-03-01\")]\n",
      "            \n",
      "        \"\"\"\n",
      "        \n",
      "        # TODO: this would be much more efficient \n",
      "        # if we first got row indicies for each period,\n",
      "        # then checked each period will fit into memory,\n",
      "        # and then iterated over the row indicies.        \n",
      "        if periods is None:\n",
      "            periods = [self.date_range()]\n",
      "        for start_date, end_date in periods:\n",
      "            yield self.load(cols, start_date, end_date)\n",
      "    \n",
      "    def estimate_memory_requirement(self, cols=None, start_date=None, end_date=None, apply_mask=True):\n",
      "        \"\"\"Returns estimated mem requirement in bytes.\"\"\"\n",
      "        BYTES_PER_ELEMENT = 4\n",
      "        BYTES_PER_TIMESTAMP = 8\n",
      "        if cols is None:\n",
      "            cols = self.column_names\n",
      "        ncols = len(cols)\n",
      "        nrows = self.nrows(start_date, end_date, apply_mask=apply_mask)\n",
      "        est_mem_usage_for_data = nrows * ncols * BYTES_PER_ELEMENT\n",
      "        est_mem_usage_for_index = nrows * BYTES_PER_TIMESTAMP\n",
      "        return est_mem_usage_for_data + est_mem_usage_for_index\n",
      "    \n",
      "    @property\n",
      "    def column_names(self):\n",
      "        storer = self._get_storer()\n",
      "        col_names = storer.non_index_axes[0][1:][0]\n",
      "        self.store.close()\n",
      "        return col_names\n",
      "    \n",
      "    def nrows(self, start_date=None, end_date=None, apply_mask=True):\n",
      "        if apply_mask:\n",
      "            start_date, end_date = self.restrict_start_and_end_dates(start_date, end_date)\n",
      "        if start_date or end_date:\n",
      "            terms = date_range_to_terms(start_date, end_date)\n",
      "            if terms == []:\n",
      "                terms = None\n",
      "            self.store.open()\n",
      "            coords = self.store.select_as_coordinates(self.table, terms)\n",
      "            nrows_ = len(coords)\n",
      "        else:\n",
      "            storer = self._get_storer()\n",
      "            nrows_ = storer.nrows\n",
      "            self.store.close()\n",
      "        return nrows_\n",
      "    \n",
      "    def restrict_start_and_end_dates(self, start_date=None, end_date=None):\n",
      "        if start_date:\n",
      "            start_date = pd.Timestamp(start_date)\n",
      "        if end_date:\n",
      "            end_date = pd.Timestamp(end_date)\n",
      "            \n",
      "        if all([start_date, self.start_date]) and start_date < self.start_date:\n",
      "            start_date = self.start_date\n",
      "        elif start_date is None:\n",
      "            start_date = self.start_date\n",
      "        if all([end_date, self.end_date]) and end_date > self.end_date:\n",
      "            end_date = self.end_date\n",
      "        elif end_date is None:\n",
      "            end_date = self.end_date\n",
      "        return start_date, end_date\n",
      "    \n",
      "    def date_range(self, apply_mask=True):\n",
      "        \"\"\"\n",
      "        Returns\n",
      "        -------\n",
      "        (start_date, end_date)\n",
      "        \"\"\"\n",
      "        self.store.open()\n",
      "        start_date = self.store.select(self.table, [0]).index[0]\n",
      "        end_date = self.store.select(self.table, start=-1).index[0]\n",
      "        self.store.close()\n",
      "        if apply_mask:\n",
      "            start_date, end_date = self.restrict_start_and_end_dates(start_date, end_date)\n",
      "        return start_date, end_date\n",
      "    \n",
      "    def _get_storer(self):\n",
      "        \"\"\"Caller must close store.\"\"\"\n",
      "        self.store.open()\n",
      "        storer = self.store.get_storer(self.table)\n",
      "        return storer\n",
      "    \n",
      "    def _check_key(self, key):\n",
      "        self.store.open()\n",
      "        keys = self.store.keys()\n",
      "        self.store.close()\n",
      "        if key not in keys:\n",
      "            raise KeyError(key + ' not in store')\n",
      "            \n",
      "    \n",
      "# SIMPLE TESTS:\n",
      "ds = HDFTableStore('store.h5', '/building1/utility/electric/meter1', end_date='2012-01-04')\n",
      "print('columns =', ds.column_names)\n",
      "print('date range (with region of interest applied) = \\n', ds.date_range())\n",
      "print('date range (with region of interest lifted) = \\n', ds.date_range(apply_mask=False))\n",
      "print('number of rows (ROI applied) =', ds.nrows())\n",
      "print('number of rows (ROI lifted) =', ds.nrows(apply_mask=False))\n",
      "print('estimated memory requirement for all data = {:.1f} MBytes'\n",
      "      .format(ds.estimate_memory_requirement() / 1E6))\n",
      "ds.load(start_date='2012-01-01 00:00:00', \n",
      "        end_date='2012-01-01 00:00:05', \n",
      "        cols=[('power', 'active')])\n",
      "for chunk in ds.get_generator([(\"2012-01-01\", \"2012-01-02\"), (\"2012-01-02\", \"2012-01-03\")]):\n",
      "    print('start = ', chunk.index[0], '; end =', chunk.index[-1])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "columns = [('power', 'active'), ('power', 'reactive'), ('voltage', '')]\n",
        "date range (with region of interest applied) = \n",
        " (Timestamp('2012-01-01 00:00:00', tz=None), Timestamp('2012-01-04 00:00:00', tz=None))\n",
        "date range (with region of interest lifted) = \n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " (Timestamp('2012-01-01 00:00:00', tz=None), Timestamp('2012-01-12 13:46:39', tz=None))\n",
        "number of rows (ROI applied) ="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 259200\n",
        "number of rows (ROI lifted) = 1000000\n",
        "estimated memory requirement for all data = 5.2 MBytes"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "start = "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 2012-01-01 00:00:00 ; end = 2012-01-01 23:59:59\n",
        "start = "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 2012-01-02 00:00:00 ; end = 2012-01-02 23:59:59\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "class TableCache(TableStore):\n",
      "    \"\"\"\n",
      "    Neither TableReader nor Meter store any data in memory persistently so,\n",
      "    without a TableCache, the following types of operation would be slow:\n",
      "    * doing multiple steps of pre-processing.  \n",
      "    * doing multiple stats (imagine having a 1GB file.  It'll be very slow if you have to re-load it from disk every time you want to do\n",
      "      another stat.)\n",
      "    \"\"\"\n",
      "    def __init__(self, table_store):\n",
      "        self.table_store = table_store\n",
      "        self.dataframe = None\n",
      "        \n",
      "    def get_generator(self):\n",
      "        \"\"\"Checks to see if any data requested is already in self.dataframe.\n",
      "        \"\"\"\n",
      "        pass"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "class Meter(object):\n",
      "    \n",
      "    def __init__(self, table_store):\n",
      "        self.table_store = table_store\n",
      "    \n",
      "    def good_chunks(self):\n",
      "        \"\"\"Returns list of (start_date, end_date) of chunks where the sample\n",
      "        period never exceeds self.max_sample_period.  \n",
      "        Caches it to self.table_reader.metadata['good_chunks'].  \n",
      "        Table_reader makes sure that metadata refers to the crop period.\n",
      "        \"\"\"\n",
      "        pass\n",
      "    \n",
      "    def dropout_rate_per_period(self):\n",
      "        chunks = self.table_reader.get_generator(period_alias='D')\n",
      "        dropout_per_period = []\n",
      "        for chunk in chunks:\n",
      "            pass # TODO\n",
      "    \n",
      "    def dropout_rate(self, ignore_gaps):\n",
      "        chunk_boundaries = self.good_chunks() if ignore_gaps else None\n",
      "        chunks = self.table_reader.get_generator(chunk_boundaries)\n",
      "        n_expected_samples = 0\n",
      "        n_actual_samples = 0\n",
      "        for chunk in chunk:\n",
      "            n_actual_samples += len(chunk)\n",
      "            n_expected_samples += (chunk.index[-1] - chunk.index[0]).total_seconds() / self.sample_period\n",
      "        dropout_rate_ = 1 - (n_actual_samples / n_expected_samples)\n",
      "        assert(0 >= dropout_rate_ <= 1, 'dropout_rate is not [0,1].  It is ' + str(dropout_rate_))\n",
      "        return dropout_rate_\n",
      "        \n",
      "    def apply_processing_funcs(self, funcs, destination=None):\n",
      "        \"\"\"\n",
      "        Parameters\n",
      "        ----------\n",
      "        funcs : list of functions\n",
      "            e.g. [normalise, bookend_good_chunks_with_zeros, remove_implausable_values]\n",
      "        destination : table_store, defaults to source\n",
      "        \"\"\"\n",
      "        if destination is None:\n",
      "            destination = self.table_store\n",
      "        chunks = self.table_reader.get_generator()\n",
      "        for chunk in chunks:\n",
      "            for func in funcs:\n",
      "                chunk = func(self, chunk)\n",
      "            table_writer.write(chunk)\n",
      "            \n",
      "        "
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}